EIA API - Data Refrsh (R Version)

Load libraries

library(dplyr)
library(EIAapi)
library(jsonlite)
library(gt)
library(plotly)
library(lubridate)
source("../R/eia_data.R")

API Settings:

meta_json <- read_json(path = "../metadata/series.json")
s <- meta_json[[1]]
series <- lapply(1:length(s), function(i) {
    return(data.frame(
        parent_id = s[[i]]$parent_id,
        parent_name = s[[i]]$parent_name,
        subba_id = s[[i]]$subba_id,
        subba_name = s[[i]]$subba_name
    ))
}) |>
    bind_rows()
api_path <- meta_json[[2]]

facets_template <- list(
    parent = NULL,
    subba = NULL
)

offset <- 2250

eia_api_key <- Sys.getenv("EIA_API_KEY")

meta_path <- "../metadata/ciso_log_R.csv"
data_path <- "../csv/ciso_grid_R.csv"

meta_obj <- get_metadata(api_key = eia_api_key, api_path = api_path, meta_path = meta_path, series = series)

gt(meta_obj$request_meta)

parent	subba	end_act	request_start	end	updates_available
CISO	PGAE	2024-08-26 07:00:00	2024-08-26 08:00:00	2025-03-07 08:00:00	TRUE
CISO	SCE	2024-08-23 07:00:00	2024-08-23 08:00:00	2025-03-07 08:00:00	TRUE
CISO	SDGE	2024-11-01 07:00:00	2024-11-01 08:00:00	2025-03-07 08:00:00	TRUE
CISO	VEA	2024-11-01 07:00:00	2024-11-01 08:00:00	2025-03-07 08:00:00	TRUE

m <- meta_obj$request_meta
index <- meta_obj$last_index + 1

data <- NULL
meta_new <- NULL

for (i in 1:nrow(m)) {
    facets <- facets_template
    facets$parent <- m$parent[i]
    facets$subba <- m$subba[i]
    start <- m$request_start[i]
    end <- m$end[i]
    print(paste(facets$parent, facets$subba, sep = " - "))

    if (m$updates_available[i]) {
        temp <- eia_backfill(
            start = start - lubridate::hours(24),
            end = end + lubridate::hours(24),
            offset = offset,
            api_key = eia_api_key,
            api_path = paste(api_path, "data", sep = ""),
            facets = facets
        ) |>
            dplyr::filter(time >= start & time <= end)
        index <- seq.POSIXt(from = start, to = end, by = "hour")
        ts_obj <- data.frame(period = index) |>
            left_join(temp, by = c("period" = "time"))
    } else {
        ts_obj <- NULL
        print("No new data is available")
    }

    meta_temp <- create_metadata(data = ts_obj, start = start, end = end, type = "refresh")

    if (is.null(ts_obj)) {
        meta_temp$parent <- m$parent[i]
        meta_temp$subba <- m$subba[i]
    }

    if (meta_temp$success) {
        print("Append the new data")
        d <- append_data(data_path = data_path, new_data = ts_obj, save = TRUE)
        meta_temp$update <- TRUE
    } else {
        meta_temp$update <- FALSE
        meta_temp$comments <- paste(meta_temp$comments, "The data refresh failed, please check the log; ", sep = "")
    }
    meta_temp$index <- NA
    meta_df <- as.data.frame(meta_temp)
    if (!is.null(ts_obj)) {
        data <- bind_rows(data, ts_obj)
    }
    meta_new <- bind_rows(meta_new, meta_df)
}

[1] "CISO - PGAE"
[1] "CISO - SCE"
[1] "CISO - SDGE"
[1] "CISO - VEA"

gt(meta_new)

index	parent	subba	time	start	end	start_act	end_act	start_match	end_match	n_obs	na	type	update	success	comments
NA	CISO	PGAE	2025-03-08 05:02:42.677664	2024-08-26 08:00:00	2025-03-07 08:00:00	2024-08-26 08:00:00	2025-03-07 08:00:00	TRUE	TRUE	4635	53	refresh	FALSE	FALSE	Missing values were found; The data refresh failed, please check the log;
NA	CISO	SCE	2025-03-08 05:02:46.731056	2024-08-23 08:00:00	2025-03-07 08:00:00	2024-08-23 08:00:00	2025-03-07 08:00:00	TRUE	TRUE	4707	62	refresh	FALSE	FALSE	Missing values were found; The data refresh failed, please check the log;
NA	CISO	SDGE	2025-03-08 05:02:49.43992	2024-11-01 08:00:00	2025-03-07 08:00:00	2024-11-03 08:00:00	2025-03-07 08:00:00	FALSE	TRUE	3026	48	refresh	FALSE	FALSE	The start argument does not match the actual; Missing values were found; The data refresh failed, please check the log;
NA	CISO	VEA	2025-03-08 05:02:52.167061	2024-11-01 08:00:00	2025-03-07 08:00:00	2024-11-03 08:00:00	2025-03-07 08:00:00	FALSE	TRUE	3026	48	refresh	FALSE	FALSE	The start argument does not match the actual; Missing values were found; The data refresh failed, please check the log;

meta_updated <- append_metadata(meta_path = meta_path, new_meta = meta_new, save = TRUE, init = FALSE)

[1] "Saving the metadata file"

Plot the Series

We will use Plotly to visualize the series:

if (!is.null(data)) {
    d <- data |> arrange(subba, period)

    p <- plot_ly(d, x = ~period, y = ~value, color = ~subba, type = "scatter", mode = "lines")

    p
} else {
    print("No new data is available")
}

data <- readr::read_csv(file = data_path, col_types = readr::cols(
    period = readr::col_datetime(format = ""),
    subba = readr::col_character(),
    subba_name = readr::col_character(),
    parent = readr::col_character(),
    parent_name = readr::col_character(),
    value = readr::col_double(),
    value_units = readr::col_character()
))

p <- plot_ly(data, x = ~period, y = ~value, color = ~subba, type = "scatter", mode = "lines")

p